<!DOCTYPE html>


<html lang="zh-CN">


<head>
  <meta charset="utf-8" />
    
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
  <title>
    K8S使用ceph-csi持久化存储之RBD.md |  
  </title>
  <meta name="generator" content="hexo-theme-ayer">
  
  <link rel="shortcut icon" href="/favicon.ico" />
  
  
<link rel="stylesheet" href="/dist/main.css">

  
<link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/Shen-Yu/cdn/css/remixicon.min.css">

  
<link rel="stylesheet" href="/css/custom.css">

  
  
<script src="https://cdn.jsdelivr.net/npm/pace-js@1.0.2/pace.min.js"></script>

  
  

  

</head>

</html>

<body>
  <div id="app">
    
      
    <main class="content on">
      <section class="outer">
  <article
  id="post-k8s/K8S使用ceph-csi持久化存储之RBD"
  class="article article-type-post"
  itemscope
  itemprop="blogPost"
  data-scroll-reveal
>
  <div class="article-inner">
    
    <header class="article-header">
       
<h1 class="article-title sea-center" style="border-left:0" itemprop="name">
  K8S使用ceph-csi持久化存储之RBD.md
</h1>
 

    </header>
     
    <div class="article-meta">
      <a href="/2020/11/11/k8s/K8S%E4%BD%BF%E7%94%A8ceph-csi%E6%8C%81%E4%B9%85%E5%8C%96%E5%AD%98%E5%82%A8%E4%B9%8BRBD/" class="article-date">
  <time datetime="2020-11-10T16:00:00.000Z" itemprop="datePublished">2020-11-11</time>
</a> 
  <div class="article-category">
    <a class="article-category-link" href="/categories/k8s/">k8s</a>
  </div>
  
<div class="word_count">
    <span class="post-time">
        <span class="post-meta-item-icon">
            <i class="ri-quill-pen-line"></i>
            <span class="post-meta-item-text"> 字数统计:</span>
            <span class="post-count">2.4k</span>
        </span>
    </span>

    <span class="post-time">
        &nbsp; | &nbsp;
        <span class="post-meta-item-icon">
            <i class="ri-book-open-line"></i>
            <span class="post-meta-item-text"> 阅读时长≈</span>
            <span class="post-count">13 分钟</span>
        </span>
    </span>
</div>
 
    </div>
      
    <div class="tocbot"></div>




  
    <div class="article-entry" itemprop="articleBody">
       
  <h1 id="K8S使用ceph-csi持久化存储之RBD"><a href="#K8S使用ceph-csi持久化存储之RBD" class="headerlink" title="K8S使用ceph-csi持久化存储之RBD"></a>K8S使用ceph-csi持久化存储之RBD</h1><h3 id="一、集群和组件版本"><a href="#一、集群和组件版本" class="headerlink" title="一、集群和组件版本"></a>一、集群和组件版本</h3><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">K8S集群：1.17.3+</span><br><span class="line">Ceph集群：Nautilus（stables）</span><br><span class="line">Ceph-CSI：release-v3.1</span><br><span class="line">snapshotter-controller：release-2.1</span><br><span class="line">Linue kernel：3.10.0-1127.19.1.el7.x86_64 +</span><br></pre></td></tr></table></figure>

<ul>
<li>镜像版本：</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">docker pull quay.io/k8scsi/csi-snapshotter:v2.1.1</span><br><span class="line">docker pull quay.io/k8scsi/csi-snapshotter:v2.1.0</span><br><span class="line">docker pull quay.io/k8scsi/csi-resizer:v0.5.0</span><br><span class="line">docker pull quay.io/k8scsi/csi-provisioner:v1.6.0</span><br><span class="line">docker pull quay.io/k8scsi/csi-node-driver-registrar:v1.3.0</span><br><span class="line">docker pull quay.io/k8scsi/csi-attacher:v2.1.1</span><br><span class="line">docker pull quay.io/cephcsi/cephcsi:v3.1-canary</span><br><span class="line">docker pull quay.io/k8scsi/snapshot-controller:v2.0.1</span><br></pre></td></tr></table></figure>

<h3 id="二、部署"><a href="#二、部署" class="headerlink" title="二、部署"></a>二、部署</h3><h4 id="1）部署Ceph-CSI"><a href="#1）部署Ceph-CSI" class="headerlink" title="1）部署Ceph-CSI"></a>1）部署Ceph-CSI</h4><h5 id="1-1）克隆代码"><a href="#1-1）克隆代码" class="headerlink" title="1.1）克隆代码"></a>1.1）克隆代码</h5><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> git <span class="built_in">clone</span> https://github.com/ceph/ceph-csi.git</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> <span class="built_in">cd</span> ceph-csi/deploy/rbd/kubernetes</span></span><br></pre></td></tr></table></figure>

<h5 id="1-2）修改yaml文件"><a href="#1-2）修改yaml文件" class="headerlink" title="1.2）修改yaml文件"></a>1.2）修改yaml文件</h5><p><em>1.2.1）修改csi-rbdplugin-provisioner.yaml和csi-rbdplugin.yaml文件，注释ceph-csi-encryption-kms-config配置：</em></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> grep <span class="string">&quot;#&quot;</span> csi-rbdplugin-provisioner.yaml</span></span><br><span class="line">          # for stable functionality replace canary with latest release version</span><br><span class="line">            #- name: ceph-csi-encryption-kms-config</span><br><span class="line">            #  mountPath: /etc/ceph-csi-encryption-kms-config/</span><br><span class="line">        #- name: ceph-csi-encryption-kms-config</span><br><span class="line">        #  configMap:</span><br><span class="line">        #    name: ceph-csi-encryption-kms-config</span><br></pre></td></tr></table></figure>

<p><em>1.2.2）配置csi-config-map.yaml文件链接ceph集群的信息</em></p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># cat csi-config-map.yaml</span></span><br><span class="line"><span class="meta">---</span></span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">ConfigMap</span></span><br><span class="line"><span class="attr">data:</span></span><br><span class="line">  <span class="attr">config.json:</span> <span class="string">|-</span></span><br><span class="line">    [</span><br><span class="line">      &#123;</span><br><span class="line">        <span class="attr">&quot;clusterID&quot;:</span> <span class="string">&quot;c7b4xxf7-c61e-4668-9xx0-82c9xx5e3696&quot;</span>,    <span class="string">//</span> <span class="string">通过ceph集群的ID</span></span><br><span class="line">        <span class="attr">&quot;monitors&quot;:</span> [</span><br><span class="line">          <span class="string">&quot;xxx.xxx.xxx.xxx:6789&quot;</span></span><br><span class="line">        ]</span><br><span class="line">      &#125;</span><br><span class="line">    ]</span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">ceph-csi-config</span></span><br></pre></td></tr></table></figure>

<p><em>1.2.3）部署rbd相关的CSI</em></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f ceph-csi/deploy/rbd/kubernetes/</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get pods</span></span><br><span class="line">csi-rbdplugin-9f8kn                             3/3     Running   0          39h</span><br><span class="line">csi-rbdplugin-pnjtn                             3/3     Running   0          39h</span><br><span class="line">csi-rbdplugin-provisioner-7f469fb84-4qqbd       6/6     Running   0          41h</span><br><span class="line">csi-rbdplugin-provisioner-7f469fb84-hkc9q       6/6     Running   5          41h</span><br><span class="line">csi-rbdplugin-provisioner-7f469fb84-vm7qm       6/6     Running   0          40h</span><br></pre></td></tr></table></figure>

<h4 id="2-快照功能需要安装快照控制器支持："><a href="#2-快照功能需要安装快照控制器支持：" class="headerlink" title="2)快照功能需要安装快照控制器支持："></a>2)快照功能需要安装快照控制器支持：</h4><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">2.1）克隆代码</span><br><span class="line"><span class="meta">#</span><span class="bash"> git <span class="built_in">clone</span> https://github.com/kubernetes-csi/external-snapshotter</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> <span class="built_in">cd</span> external-snapshotter/deploy/kubernetes/snapshot-controller</span></span><br><span class="line">2.2）部署</span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl external-snapshotter/deploy/kubernetes/snapshot-controller/</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get pods | grep snapshot-controller</span></span><br><span class="line">snapshot-controller-0                           1/1     Running   0          20h</span><br><span class="line">2.3）部署crd</span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f external-snapshotter/config/crd/</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl api-versions | grep snapshot</span></span><br><span class="line">snapshot.storage.k8s.io/v1beta1</span><br></pre></td></tr></table></figure>

<p><em>至此，Ceph-CSI和snapshot-controller安装完成。下面进行功能测试。测试功能前需要在ceph集群中创建对应的存储池：</em></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br></pre></td><td class="code"><pre><span class="line">// 查看集群状态</span><br><span class="line"><span class="meta">#</span><span class="bash"> ceph -s</span></span><br><span class="line">  cluster:</span><br><span class="line">    id:     c7b43ef7-c61e-4668-9970-82c9775e3696</span><br><span class="line">    health: HEALTH_OK</span><br><span class="line"> </span><br><span class="line">  services:</span><br><span class="line">    mon: 1 daemons, quorum cka-node-01 (age 24h)</span><br><span class="line">    mgr: cka-node-01(active, since 24h), standbys: cka-node-02, cka-node-03</span><br><span class="line">    mds: cephfs:1 &#123;0=cka-node-01=up:active&#125; 2 up:standby</span><br><span class="line">    osd: 3 osds: 3 up, 3 in</span><br><span class="line">    rgw: 1 daemon active (cka-node-01)</span><br><span class="line"> </span><br><span class="line">  task status:</span><br><span class="line">    scrub status:</span><br><span class="line">        mds.cka-node-01: idle</span><br><span class="line"> </span><br><span class="line">  data:</span><br><span class="line">    pools:   7 pools, 184 pgs</span><br><span class="line">    objects: 827 objects, 1.7 GiB</span><br><span class="line">    usage:   8.1 GiB used, 52 GiB / 60 GiB avail</span><br><span class="line">    pgs:     184 active+clean</span><br><span class="line"> </span><br><span class="line">  io:</span><br><span class="line">    client:   32 KiB/s rd, 0 B/s wr, 31 op/s rd, 21 op/s wr</span><br><span class="line"> </span><br><span class="line">// 创建存储池kubernetes</span><br><span class="line"><span class="meta">#</span><span class="bash"> ceph osd pool create kubernetes 8 8</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> rbd pool init kubernetes</span></span><br><span class="line"> </span><br><span class="line">// 创建用户kubernetes</span><br><span class="line"><span class="meta">#</span><span class="bash"> ceph auth get-or-create client.kubernetes mon <span class="string">&#x27;profile rbd&#x27;</span> osd <span class="string">&#x27;profile rbd pool=kubernetes&#x27;</span></span></span><br><span class="line"> </span><br><span class="line">// 获取集群信息和查看用户key</span><br><span class="line"><span class="meta">#</span><span class="bash"> ceph mon dump</span></span><br><span class="line">dumped monmap epoch 3</span><br><span class="line">epoch 3</span><br><span class="line">fsid c7b43ef7-c61e-4668-9970-82c9775e3696</span><br><span class="line">last_changed 2020-09-11 11:05:25.529648</span><br><span class="line">created 2020-09-10 16:22:52.967856</span><br><span class="line">min_mon_release 14 (nautilus)</span><br><span class="line">0: [v2:10.0.xxx.xxx0:3300/0,v1:10.0.xxx.xxx:6789/0] mon.cka-node-01</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> ceph auth get client.kubernetes</span></span><br><span class="line">exported keyring for client.kubernetes</span><br><span class="line">[client.kubernetes]</span><br><span class="line">    key = AQBt5xxxR0DBAAtjxxA+zlqxxxF3shYm8qLQmw==</span><br><span class="line">    caps mon = &quot;profile rbd&quot;</span><br><span class="line">    caps osd = &quot;profile rbd pool=kubernetes&quot;</span><br></pre></td></tr></table></figure>

<h3 id="三、验证"><a href="#三、验证" class="headerlink" title="三、验证"></a>三、验证</h3><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">验证如下功能：</span><br><span class="line">1）创建rbd类型pvc给pod使用；</span><br><span class="line">2）创建rbd类型pvc的快照，并验证基于快照恢复的可用性；</span><br><span class="line">3）扩容pvc大小；</span><br><span class="line">4）同一个pvc重复创建快照；</span><br></pre></td></tr></table></figure>

<h5 id="1、创建rbd类型pvc给pod使用："><a href="#1、创建rbd类型pvc给pod使用：" class="headerlink" title="1、创建rbd类型pvc给pod使用："></a>1、创建rbd类型pvc给pod使用：</h5><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">1.1</span><span class="string">)</span> <span class="string">创建连接ceph集群的秘钥</span></span><br><span class="line"><span class="comment"># cat secret.yaml</span></span><br><span class="line"><span class="meta">---</span></span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Secret</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">csi-rbd-secret</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line"><span class="attr">stringData:</span></span><br><span class="line">  <span class="attr">userID:</span> <span class="string">kubernetes</span></span><br><span class="line">  <span class="attr">userKey:</span> <span class="string">AQBt51lf9iR0DBAAtjA+zlqxxxYm8qLQmw==</span></span><br><span class="line">  <span class="attr">encryptionPassphrase:</span> <span class="string">test_passphrase</span></span><br><span class="line"> </span><br><span class="line"><span class="comment"># kubectl apply -f secret.yaml</span></span><br><span class="line"><span class="number">1.2</span><span class="string">)</span> <span class="string">创建storeclass</span></span><br><span class="line"><span class="comment"># cat storageclass.yaml</span></span><br><span class="line"><span class="meta">---</span></span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">storage.k8s.io/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">StorageClass</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">   <span class="attr">name:</span> <span class="string">csi-rbd-sc</span></span><br><span class="line"><span class="attr">provisioner:</span> <span class="string">rbd.csi.ceph.com</span></span><br><span class="line"><span class="attr">parameters:</span></span><br><span class="line">   <span class="attr">clusterID:</span> <span class="string">c7b43xxf7-c61e-4668-9970-82c9e3696</span></span><br><span class="line">   <span class="attr">pool:</span> <span class="string">kubernetes</span></span><br><span class="line">   <span class="attr">imageFeatures:</span> <span class="string">layering</span></span><br><span class="line">   <span class="attr">csi.storage.k8s.io/provisioner-secret-name:</span> <span class="string">csi-rbd-secret</span></span><br><span class="line">   <span class="attr">csi.storage.k8s.io/provisioner-secret-namespace:</span> <span class="string">default</span></span><br><span class="line">   <span class="attr">csi.storage.k8s.io/controller-expand-secret-name:</span> <span class="string">csi-rbd-secret</span></span><br><span class="line">   <span class="attr">csi.storage.k8s.io/controller-expand-secret-namespace:</span> <span class="string">default</span></span><br><span class="line">   <span class="attr">csi.storage.k8s.io/node-stage-secret-name:</span> <span class="string">csi-rbd-secret</span></span><br><span class="line">   <span class="attr">csi.storage.k8s.io/node-stage-secret-namespace:</span> <span class="string">default</span></span><br><span class="line">   <span class="attr">csi.storage.k8s.io/fstype:</span> <span class="string">ext4</span></span><br><span class="line"><span class="attr">reclaimPolicy:</span> <span class="string">Delete</span></span><br><span class="line"><span class="attr">allowVolumeExpansion:</span> <span class="literal">true</span></span><br><span class="line"><span class="attr">mountOptions:</span></span><br><span class="line">   <span class="bullet">-</span> <span class="string">discard</span></span><br><span class="line"> </span><br><span class="line"><span class="comment"># kubectl apply -f storageclass.yaml</span></span><br><span class="line"><span class="number">1.3</span><span class="string">)基于storeclass创建pvc</span></span><br><span class="line"><span class="comment"># cat pvc.yaml</span></span><br><span class="line"><span class="meta">---</span></span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">PersistentVolumeClaim</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">rbd-pvc</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">accessModes:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="string">ReadWriteOnce</span></span><br><span class="line">  <span class="attr">resources:</span></span><br><span class="line">    <span class="attr">requests:</span></span><br><span class="line">      <span class="attr">storage:</span> <span class="string">1Gi</span></span><br><span class="line">  <span class="attr">storageClassName:</span> <span class="string">csi-rbd-sc</span></span><br><span class="line"> </span><br><span class="line"><span class="comment"># kubectl apply -f pvc.yaml</span></span><br><span class="line"><span class="comment"># kubectl get pvc rbd-pvc</span></span><br><span class="line"><span class="string">NAME</span>      <span class="string">STATUS</span>   <span class="string">VOLUME</span>                                     <span class="string">CAPACITY</span>   <span class="string">ACCESS</span> <span class="string">MODES</span>   <span class="string">STORAGECLASS</span>   <span class="string">AGE</span></span><br><span class="line"><span class="string">rbd-pvc</span>   <span class="string">Bound</span>    <span class="string">pvc-11b931b0-7cb5-40e1-815b-c15659310593</span>   <span class="string">1Gi</span>      <span class="string">RWO</span>            <span class="string">csi-rbd-sc</span>        <span class="string">17h</span></span><br><span class="line"><span class="number">1.4</span><span class="string">）创建pod应用pvc</span></span><br><span class="line"><span class="comment"># cat pod.yaml</span></span><br><span class="line"><span class="meta">---</span></span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">csi-rbd-demo-pod</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">web-server</span></span><br><span class="line">      <span class="attr">image:</span> <span class="string">nginx</span></span><br><span class="line">      <span class="attr">volumeMounts:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">mypvc</span></span><br><span class="line">          <span class="attr">mountPath:</span> <span class="string">/var/lib/www/html</span></span><br><span class="line">  <span class="attr">volumes:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">mypvc</span></span><br><span class="line">      <span class="attr">persistentVolumeClaim:</span></span><br><span class="line">        <span class="attr">claimName:</span> <span class="string">rbd-pvc</span></span><br><span class="line">        <span class="attr">readOnly:</span> <span class="literal">false</span></span><br><span class="line"> </span><br><span class="line"><span class="comment"># kubectl apply -f pod.yaml</span></span><br><span class="line"><span class="comment"># kubectl get pods csi-rbd-demo-pod</span></span><br><span class="line"><span class="string">NAME</span>               <span class="string">READY</span>   <span class="string">STATUS</span>    <span class="string">RESTARTS</span>   <span class="string">AGE</span></span><br><span class="line"><span class="string">csi-rbd-demo-pod</span>   <span class="number">1</span><span class="string">/1</span>     <span class="string">Running</span>   <span class="number">0</span>          <span class="string">40h</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># kubectl exec -ti csi-rbd-demo-pod -- bash</span></span><br><span class="line"><span class="string">root@csi-rbd-demo-pod:/#</span> <span class="string">df</span> <span class="string">-h</span></span><br><span class="line"><span class="string">Filesystem</span>               <span class="string">Size</span>  <span class="string">Used</span> <span class="string">Avail</span> <span class="string">Use%</span> <span class="string">Mounted</span> <span class="string">on</span></span><br><span class="line"><span class="string">overlay</span>                  <span class="string">199G</span>  <span class="number">7.</span><span class="string">4G</span>  <span class="string">192G</span>   <span class="number">4</span><span class="string">%</span> <span class="string">/</span></span><br><span class="line"><span class="string">tmpfs</span>                     <span class="string">64M</span>     <span class="number">0</span>   <span class="string">64M</span>   <span class="number">0</span><span class="string">%</span> <span class="string">/dev</span></span><br><span class="line"><span class="string">tmpfs</span>                    <span class="number">7.</span><span class="string">8G</span>     <span class="number">0</span>  <span class="number">7.</span><span class="string">8G</span>   <span class="number">0</span><span class="string">%</span> <span class="string">/sys/fs/cgroup</span></span><br><span class="line"><span class="string">/dev/mapper/centos-root</span>  <span class="string">199G</span>  <span class="number">7.</span><span class="string">4G</span>  <span class="string">192G</span>   <span class="number">4</span><span class="string">%</span> <span class="string">/etc/hosts</span></span><br><span class="line"><span class="string">shm</span>                       <span class="string">64M</span>     <span class="number">0</span>   <span class="string">64M</span>   <span class="number">0</span><span class="string">%</span> <span class="string">/dev/shm</span></span><br><span class="line"><span class="string">/dev/rbd0</span>                <span class="string">976M</span>  <span class="number">2.</span><span class="string">6M</span>  <span class="string">958M</span>   <span class="number">1</span><span class="string">%</span> <span class="string">/var/lib/www/html</span></span><br><span class="line"><span class="string">tmpfs</span>                    <span class="number">7.</span><span class="string">8G</span>   <span class="string">12K</span>  <span class="number">7.</span><span class="string">8G</span>   <span class="number">1</span><span class="string">%</span> <span class="string">/run/secrets/kubernetes.io/serviceaccount</span></span><br><span class="line"><span class="string">tmpfs</span>                    <span class="number">7.</span><span class="string">8G</span>     <span class="number">0</span>  <span class="number">7.</span><span class="string">8G</span>   <span class="number">0</span><span class="string">%</span> <span class="string">/proc/acpi</span></span><br><span class="line"><span class="string">tmpfs</span>                    <span class="number">7.</span><span class="string">8G</span>     <span class="number">0</span>  <span class="number">7.</span><span class="string">8G</span>   <span class="number">0</span><span class="string">%</span> <span class="string">/proc/scsi</span></span><br><span class="line"><span class="string">tmpfs</span>                    <span class="number">7.</span><span class="string">8G</span>     <span class="number">0</span>  <span class="number">7.</span><span class="string">8G</span>   <span class="number">0</span><span class="string">%</span> <span class="string">/sys/firmware</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 写入文件，用于后续快照验证</span></span><br><span class="line"><span class="string">root@csi-rbd-demo-pod:/#</span> <span class="string">cd</span> <span class="string">/var/lib/www/html;mkdir</span> <span class="string">demo;cd</span> <span class="string">demo;echo</span> <span class="string">&quot;snapshot test&quot;</span> <span class="string">&gt;</span> <span class="string">test.txt</span></span><br><span class="line"><span class="string">root@csi-rbd-demo-pod:/var/lib/www/html#</span> <span class="string">cat</span> <span class="string">demo/test.txt</span></span><br><span class="line"><span class="string">snapshot</span> <span class="string">test</span></span><br></pre></td></tr></table></figure>

<h5 id="2）创建rbd类型pvc的快照，并验证基于快照恢复的可用性："><a href="#2）创建rbd类型pvc的快照，并验证基于快照恢复的可用性：" class="headerlink" title="2）创建rbd类型pvc的快照，并验证基于快照恢复的可用性："></a>2）创建rbd类型pvc的快照，并验证基于快照恢复的可用性：</h5><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">2.1</span><span class="string">)创建上一步pvc的快照</span></span><br><span class="line"><span class="comment"># cat snapshot.yaml</span></span><br><span class="line"><span class="meta">---</span></span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">snapshot.storage.k8s.io/v1beta1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">VolumeSnapshot</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">rbd-pvc-snapshot</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">volumeSnapshotClassName:</span> <span class="string">csi-rbdplugin-snapclass</span></span><br><span class="line">  <span class="attr">source:</span></span><br><span class="line">    <span class="attr">persistentVolumeClaimName:</span> <span class="string">rbd-pvc</span></span><br><span class="line"> </span><br><span class="line"><span class="comment"># kubectl apply -f snapshot.yaml</span></span><br><span class="line"><span class="comment"># kubectl get VolumeSnapshot rbd-pvc-snapshot</span></span><br><span class="line"><span class="string">NAME</span>               <span class="string">READYTOUSE</span>   <span class="string">SOURCEPVC</span>   <span class="string">SOURCESNAPSHOTCONTENT</span>   <span class="string">RESTORESIZE</span>   <span class="string">SNAPSHOTCLASS</span>             <span class="string">SNAPSHOTCONTENT</span>                                    <span class="string">CREATIONTIME</span>   <span class="string">AGE</span></span><br><span class="line"><span class="string">rbd-pvc-snapshot</span>   <span class="literal">true</span>         <span class="string">rbd-pvc</span>                             <span class="string">1Gi</span>           <span class="string">csi-rbdplugin-snapclass</span>   <span class="string">snapcontent-48f3e563-d21a-40bb-8e15-ddbf27886c88</span>   <span class="string">19h</span>            <span class="string">19h</span></span><br><span class="line"><span class="number">2.2</span><span class="string">)创建基于快照恢复的pvc</span></span><br><span class="line"><span class="comment"># cat pvc-restore.yaml</span></span><br><span class="line"><span class="meta">---</span></span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">PersistentVolumeClaim</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">rbd-pvc-restore</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">storageClassName:</span> <span class="string">csi-rbd-sc</span></span><br><span class="line">  <span class="attr">dataSource:</span></span><br><span class="line">    <span class="attr">name:</span> <span class="string">rbd-pvc-snapshot</span></span><br><span class="line">    <span class="attr">kind:</span> <span class="string">VolumeSnapshot</span></span><br><span class="line">    <span class="attr">apiGroup:</span> <span class="string">snapshot.storage.k8s.io</span></span><br><span class="line">  <span class="attr">accessModes:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="string">ReadWriteOnce</span></span><br><span class="line">  <span class="attr">resources:</span></span><br><span class="line">    <span class="attr">requests:</span></span><br><span class="line">      <span class="attr">storage:</span> <span class="string">1Gi</span></span><br><span class="line"> </span><br><span class="line"><span class="comment"># kubectl apply -f pvc-restore.yaml</span></span><br><span class="line"><span class="number">2.3</span><span class="string">)创建pod应用快照恢复的pvc</span></span><br><span class="line"><span class="comment"># cat pod-restore.yaml</span></span><br><span class="line"><span class="meta">---</span></span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">csi-rbd-restore-demo-pod</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">web-server</span></span><br><span class="line">      <span class="attr">image:</span> <span class="string">nginx</span></span><br><span class="line">      <span class="attr">volumeMounts:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">mypvc</span></span><br><span class="line">          <span class="attr">mountPath:</span> <span class="string">/var/lib/www/html</span></span><br><span class="line">  <span class="attr">volumes:</span></span><br><span class="line">    <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">mypvc</span></span><br><span class="line">      <span class="attr">persistentVolumeClaim:</span></span><br><span class="line">        <span class="attr">claimName:</span> <span class="string">rbd-pvc-restore</span></span><br><span class="line">        <span class="attr">readOnly:</span> <span class="literal">false</span></span><br><span class="line"> </span><br><span class="line"><span class="comment"># kubectl apply -f pod-restore.yaml</span></span><br><span class="line"><span class="comment"># kubectl get pods csi-rbd-restore-demo-pod</span></span><br><span class="line"><span class="string">NAME</span>                       <span class="string">READY</span>   <span class="string">STATUS</span>    <span class="string">RESTARTS</span>   <span class="string">AGE</span></span><br><span class="line"><span class="string">csi-rbd-restore-demo-pod</span>   <span class="number">1</span><span class="string">/1</span>     <span class="string">Running</span>   <span class="number">0</span>          <span class="string">18h</span></span><br><span class="line"><span class="comment"># kubectl exec -ti csi-rbd-restore-demo-pod -- bash</span></span><br><span class="line"><span class="string">root@csi-rbd-restore-demo-pod:/#</span> <span class="string">df</span> <span class="string">-h</span></span><br><span class="line"><span class="string">Filesystem</span>               <span class="string">Size</span>  <span class="string">Used</span> <span class="string">Avail</span> <span class="string">Use%</span> <span class="string">Mounted</span> <span class="string">on</span></span><br><span class="line"><span class="string">overlay</span>                  <span class="string">199G</span>  <span class="number">7.</span><span class="string">4G</span>  <span class="string">192G</span>   <span class="number">4</span><span class="string">%</span> <span class="string">/</span></span><br><span class="line"><span class="string">tmpfs</span>                     <span class="string">64M</span>     <span class="number">0</span>   <span class="string">64M</span>   <span class="number">0</span><span class="string">%</span> <span class="string">/dev</span></span><br><span class="line"><span class="string">tmpfs</span>                    <span class="number">7.</span><span class="string">8G</span>     <span class="number">0</span>  <span class="number">7.</span><span class="string">8G</span>   <span class="number">0</span><span class="string">%</span> <span class="string">/sys/fs/cgroup</span></span><br><span class="line"><span class="string">/dev/mapper/centos-root</span>  <span class="string">199G</span>  <span class="number">7.</span><span class="string">4G</span>  <span class="string">192G</span>   <span class="number">4</span><span class="string">%</span> <span class="string">/etc/hosts</span></span><br><span class="line"><span class="string">shm</span>                       <span class="string">64M</span>     <span class="number">0</span>   <span class="string">64M</span>   <span class="number">0</span><span class="string">%</span> <span class="string">/dev/shm</span></span><br><span class="line"><span class="string">/dev/rbd3</span>                <span class="string">976M</span>  <span class="number">2.</span><span class="string">6M</span>  <span class="string">958M</span>   <span class="number">1</span><span class="string">%</span> <span class="string">/var/lib/www/html</span></span><br><span class="line"><span class="string">tmpfs</span>                    <span class="number">7.</span><span class="string">8G</span>   <span class="string">12K</span>  <span class="number">7.</span><span class="string">8G</span>   <span class="number">1</span><span class="string">%</span> <span class="string">/run/secrets/kubernetes.io/serviceaccount</span></span><br><span class="line"><span class="string">tmpfs</span>                    <span class="number">7.</span><span class="string">8G</span>     <span class="number">0</span>  <span class="number">7.</span><span class="string">8G</span>   <span class="number">0</span><span class="string">%</span> <span class="string">/proc/acpi</span></span><br><span class="line"><span class="string">tmpfs</span>                    <span class="number">7.</span><span class="string">8G</span>     <span class="number">0</span>  <span class="number">7.</span><span class="string">8G</span>   <span class="number">0</span><span class="string">%</span> <span class="string">/proc/scsi</span></span><br><span class="line"><span class="string">tmpfs</span>                    <span class="number">7.</span><span class="string">8G</span>     <span class="number">0</span>  <span class="number">7.</span><span class="string">8G</span>   <span class="number">0</span><span class="string">%</span> <span class="string">/sys/firmware</span></span><br><span class="line"></span><br><span class="line"><span class="string">root@csi-rbd-restore-demo-pod:/#</span> <span class="string">cd</span> <span class="string">/var/lib/www/html</span></span><br><span class="line"><span class="string">root@csi-rbd-restore-demo-pod:/var/lib/www/html#</span> <span class="string">ls</span></span><br><span class="line"><span class="string">demo</span>  <span class="string">lost+found</span></span><br><span class="line"><span class="string">root@csi-rbd-restore-demo-pod:/var/lib/www/html#</span> <span class="string">cat</span> <span class="string">demo/test.txt</span></span><br><span class="line"><span class="string">snapshot</span> <span class="string">test</span></span><br><span class="line"> </span><br><span class="line"><span class="string">//基于快照恢复数据功能正常</span></span><br></pre></td></tr></table></figure>

<h5 id="3）扩容pvc大小："><a href="#3）扩容pvc大小：" class="headerlink" title="3）扩容pvc大小："></a>3）扩容pvc大小：</h5><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br></pre></td><td class="code"><pre><span class="line">3.1)修改rbd-pvc的容量大小</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat pvc.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: PersistentVolumeClaim</span><br><span class="line">metadata:</span><br><span class="line">  name: rbd-pvc</span><br><span class="line">spec:</span><br><span class="line">  accessModes:</span><br><span class="line">    - ReadWriteOnce</span><br><span class="line">  resources:</span><br><span class="line">    requests:</span><br><span class="line">      storage: 100Gi    // 由1G改为100G</span><br><span class="line">  storageClassName: csi-rbd-sc</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f pvc.yaml</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get pvc rbd-pvc</span></span><br><span class="line">NAME      STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE</span><br><span class="line">rbd-pvc   Bound    pvc-11b931b0-7cb5-40e1-815b-c15659310593   100Gi      RWO            csi-rbd-sc     40h</span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl <span class="built_in">exec</span> -ti csi-rbd-demo-pod -- bash</span></span><br><span class="line">root@csi-rbd-demo-pod:/# df -h</span><br><span class="line">Filesystem               Size  Used Avail Use% Mounted on</span><br><span class="line">overlay                  199G  7.4G  192G   4% /</span><br><span class="line">tmpfs                     64M     0   64M   0% /dev</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /sys/fs/cgroup</span><br><span class="line">/dev/mapper/centos-root  199G  7.4G  192G   4% /etc/hosts</span><br><span class="line">shm                       64M     0   64M   0% /dev/shm</span><br><span class="line">/dev/rbd0                 99G  6.8M   99G   1% /var/lib/www/html    // 扩容正常</span><br><span class="line">tmpfs                    7.8G   12K  7.8G   1% /run/secrets/kubernetes.io/serviceaccount</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /proc/acpi</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /proc/scsi</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /sys/firmware</span><br><span class="line"> </span><br><span class="line">// 再次写入数据用于后续第二次创建快照</span><br><span class="line">root@csi-rbd-demo-pod:# cd /var/lib/www/html;mkdir test;echo &quot;abc&quot; &gt; test/demo.txt;echo &quot;abc&quot; &gt;&gt; /var/lib/www/html/demo/test.txt</span><br><span class="line">root@csi-rbd-demo-pod:/var/lib/www/html# cat test/demo.txt</span><br><span class="line">abc</span><br><span class="line">root@csi-rbd-demo-pod:/var/lib/www/html# cat demo/test.txt</span><br><span class="line">snapshot test</span><br><span class="line">abc</span><br></pre></td></tr></table></figure>

<h5 id="4）同一个pvc重复创建快照："><a href="#4）同一个pvc重复创建快照：" class="headerlink" title="4）同一个pvc重复创建快照："></a>4）同一个pvc重复创建快照：</h5><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br></pre></td><td class="code"><pre><span class="line">4.1)再次对rbd-pvc创建快照</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat snapshot-1.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: snapshot.storage.k8s.io/v1beta1</span><br><span class="line">kind: VolumeSnapshot</span><br><span class="line">metadata:</span><br><span class="line">  name: rbd-pvc-snapshot-1</span><br><span class="line">spec:</span><br><span class="line">  volumeSnapshotClassName: csi-rbdplugin-snapclass</span><br><span class="line">  source:</span><br><span class="line">    persistentVolumeClaimName: rbd-pvc</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f snapshot-1.yaml</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get VolumeSnapshot rbd-pvc-snapshot-1</span></span><br><span class="line">NAME                 READYTOUSE   SOURCEPVC   SOURCESNAPSHOTCONTENT   RESTORESIZE   SNAPSHOTCLASS             SNAPSHOTCONTENT                                    CREATIONTIME   AGE</span><br><span class="line">rbd-pvc-snapshot-1   true         rbd-pvc                             100Gi         csi-rbdplugin-snapclass   snapcontent-b82dceb0-7ba6-4a3e-88ab-2220b729d85f   18h            18h</span><br><span class="line">4.2)基于rbd-pvc-snapshot-1快照恢复pvc</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat pvc-restore-1.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: PersistentVolumeClaim</span><br><span class="line">metadata:</span><br><span class="line">  name: rbd-pvc-restore-1</span><br><span class="line">spec:</span><br><span class="line">  storageClassName: csi-rbd-sc</span><br><span class="line">  dataSource:</span><br><span class="line">    name: rbd-pvc-snapshot-1</span><br><span class="line">    kind: VolumeSnapshot</span><br><span class="line">    apiGroup: snapshot.storage.k8s.io</span><br><span class="line">  accessModes:</span><br><span class="line">    - ReadWriteOnce</span><br><span class="line">  resources:</span><br><span class="line">    requests:</span><br><span class="line">      storage: 100Gi</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f pvc-restore-1.yaml</span></span><br><span class="line">4.3)创建pod引用rbd-pvc-restore-1恢复的pvc</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat pod-restore-1.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Pod</span><br><span class="line">metadata:</span><br><span class="line">  name: csi-rbd-restore-demo-pod-1</span><br><span class="line">spec:</span><br><span class="line">  containers:</span><br><span class="line">    - name: web-server</span><br><span class="line">      image: nginx</span><br><span class="line">      volumeMounts:</span><br><span class="line">        - name: mypvc</span><br><span class="line">          mountPath: /var/lib/www/html</span><br><span class="line">  volumes:</span><br><span class="line">    - name: mypvc</span><br><span class="line">      persistentVolumeClaim:</span><br><span class="line">        claimName: rbd-pvc-restore-1</span><br><span class="line">        readOnly: false</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f pod-restore-1.yaml</span></span><br><span class="line">NAME                         READY   STATUS    RESTARTS   AGE</span><br><span class="line">csi-rbd-restore-demo-pod-1   1/1     Running   0          18h</span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl <span class="built_in">exec</span> -ti csi-rbd-restore-demo-pod-1 -- bash</span></span><br><span class="line">root@csi-rbd-restore-demo-pod-1:/# df -h</span><br><span class="line">Filesystem               Size  Used Avail Use% Mounted on</span><br><span class="line">overlay                  199G  7.4G  192G   4% /</span><br><span class="line">tmpfs                     64M     0   64M   0% /dev</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /sys/fs/cgroup</span><br><span class="line">/dev/mapper/centos-root  199G  7.4G  192G   4% /etc/hosts</span><br><span class="line">shm                       64M     0   64M   0% /dev/shm</span><br><span class="line">/dev/rbd4                 99G  6.8M   99G   1% /var/lib/www/html</span><br><span class="line">tmpfs                    7.8G   12K  7.8G   1% /run/secrets/kubernetes.io/serviceaccount</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /proc/acpi</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /proc/scsi</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /sys/firmware</span><br><span class="line">root@csi-rbd-restore-demo-pod-1:/# cd /var/lib/www/html</span><br><span class="line">root@csi-rbd-restore-demo-pod-1:/var/lib/www/html# cat demo/test.txt</span><br><span class="line">snapshot test</span><br><span class="line">abc</span><br><span class="line">root@csi-rbd-restore-demo-pod-1:/var/lib/www/html# cat test/demo.txt</span><br><span class="line">abc</span><br><span class="line"> </span><br><span class="line">// 至此验证扩容后的pvc，二次创建的快照恢复数据功能正常</span><br><span class="line"> </span><br><span class="line">// 查看第一个创建的快照中是否有后续添加的文件数据,如下数据还是第一个快照创建时数据</span><br><span class="line">[root@cka-node-01 rbd]# kubectl exec -ti csi-rbd-restore-demo-pod -- bash</span><br><span class="line">root@csi-rbd-restore-demo-pod:/# df -h</span><br><span class="line">Filesystem               Size  Used Avail Use% Mounted on</span><br><span class="line">overlay                  199G  7.4G  192G   4% /</span><br><span class="line">tmpfs                     64M     0   64M   0% /dev</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /sys/fs/cgroup</span><br><span class="line">/dev/mapper/centos-root  199G  7.4G  192G   4% /etc/hosts</span><br><span class="line">shm                       64M     0   64M   0% /dev/shm</span><br><span class="line">/dev/rbd3                976M  2.6M  958M   1% /var/lib/www/html</span><br><span class="line">tmpfs                    7.8G   12K  7.8G   1% /run/secrets/kubernetes.io/serviceaccount</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /proc/acpi</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /proc/scsi</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /sys/firmware</span><br><span class="line">root@csi-rbd-restore-demo-pod:/# cd /var/lib/www/html</span><br><span class="line">root@csi-rbd-restore-demo-pod:/var/lib/www/html# cat demo/test.txt</span><br><span class="line">snapshot test</span><br><span class="line">root@csi-rbd-restore-demo-pod:/var/lib/www/html# ls</span><br><span class="line">demo  lost+found</span><br></pre></td></tr></table></figure>

<p><em>完</em></p>
<p>END</p>
 
      <!-- reward -->
      
      <div id="reword-out">
        <div id="reward-btn">
          打赏
        </div>
      </div>
      
    </div>
    

    <!-- copyright -->
    
    <div class="declare">
      <ul class="post-copyright">
        <li>
          <i class="ri-copyright-line"></i>
          <strong>版权声明： </strong>
          
          本博客所有文章除特别声明外，著作权归作者所有。转载请注明出处！
          
        </li>
      </ul>
    </div>
    
    <footer class="article-footer">
       
<div class="share-btn">
      <span class="share-sns share-outer">
        <i class="ri-share-forward-line"></i>
        分享
      </span>
      <div class="share-wrap">
        <i class="arrow"></i>
        <div class="share-icons">
          
          <a class="weibo share-sns" href="javascript:;" data-type="weibo">
            <i class="ri-weibo-fill"></i>
          </a>
          <a class="weixin share-sns wxFab" href="javascript:;" data-type="weixin">
            <i class="ri-wechat-fill"></i>
          </a>
          <a class="qq share-sns" href="javascript:;" data-type="qq">
            <i class="ri-qq-fill"></i>
          </a>
          <a class="douban share-sns" href="javascript:;" data-type="douban">
            <i class="ri-douban-line"></i>
          </a>
          <!-- <a class="qzone share-sns" href="javascript:;" data-type="qzone">
            <i class="icon icon-qzone"></i>
          </a> -->
          
          <a class="facebook share-sns" href="javascript:;" data-type="facebook">
            <i class="ri-facebook-circle-fill"></i>
          </a>
          <a class="twitter share-sns" href="javascript:;" data-type="twitter">
            <i class="ri-twitter-fill"></i>
          </a>
          <a class="google share-sns" href="javascript:;" data-type="google">
            <i class="ri-google-fill"></i>
          </a>
        </div>
      </div>
</div>

<div class="wx-share-modal">
    <a class="modal-close" href="javascript:;"><i class="ri-close-circle-line"></i></a>
    <p>扫一扫，分享到微信</p>
    <div class="wx-qrcode">
      <img src="//api.qrserver.com/v1/create-qr-code/?size=150x150&data=http://example.com/2020/11/11/k8s/K8S%E4%BD%BF%E7%94%A8ceph-csi%E6%8C%81%E4%B9%85%E5%8C%96%E5%AD%98%E5%82%A8%E4%B9%8BRBD/" alt="微信分享二维码">
    </div>
</div>

<div id="share-mask"></div>  
  <ul class="article-tag-list" itemprop="keywords"><li class="article-tag-list-item"><a class="article-tag-list-link" href="/tags/k8s/" rel="tag">k8s</a></li></ul>

    </footer>
  </div>

   
  <nav class="article-nav">
    
      <a href="/2020/11/11/docker/%E8%A7%A3%E5%AF%86%20Docker%20%E6%8C%82%E8%BD%BD%E6%96%87%E4%BB%B6%EF%BC%8C%E5%AE%BF%E4%B8%BB%E6%9C%BA%E4%BF%AE%E6%94%B9%E5%90%8E%E5%AE%B9%E5%99%A8%E9%87%8C%E6%96%87%E4%BB%B6%E6%B2%A1%E6%9C%89%E4%BF%AE%E6%94%B9/" class="article-nav-link">
        <strong class="article-nav-caption">上一篇</strong>
        <div class="article-nav-title">
          
            解密 Docker 挂载文件，宿主机修改后容器里文件没有修改.md
          
        </div>
      </a>
    
    
      <a href="/2020/11/11/k8s/Kubectl%E8%BF%9C%E7%A8%8B%E8%BF%9E%E6%8E%A5%E9%9B%86%E7%BE%A4/" class="article-nav-link">
        <strong class="article-nav-caption">下一篇</strong>
        <div class="article-nav-title">Kubectl远程连接集群.md</div>
      </a>
    
  </nav>

   
<!-- valine评论 -->
<div id="vcomments-box">
  <div id="vcomments"></div>
</div>
<script src="//cdn1.lncld.net/static/js/3.0.4/av-min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/valine@1.4.14/dist/Valine.min.js"></script>
<script>
  new Valine({
    el: "#vcomments",
    app_id: "",
    app_key: "",
    path: window.location.pathname,
    avatar: "monsterid",
    placeholder: "给我的文章加点评论吧~",
    recordIP: true,
  });
  const infoEle = document.querySelector("#vcomments .info");
  if (infoEle && infoEle.childNodes && infoEle.childNodes.length > 0) {
    infoEle.childNodes.forEach(function (item) {
      item.parentNode.removeChild(item);
    });
  }
</script>
<style>
  #vcomments-box {
    padding: 5px 30px;
  }

  @media screen and (max-width: 800px) {
    #vcomments-box {
      padding: 5px 0px;
    }
  }

  #vcomments-box #vcomments {
    background-color: #fff;
  }

  .v .vlist .vcard .vh {
    padding-right: 20px;
  }

  .v .vlist .vcard {
    padding-left: 10px;
  }
</style>

 
     
</article>

</section>
      <footer class="footer">
  <div class="outer">
    <ul>
      <li>
        Copyrights &copy;
        2015-2020
        <i class="ri-heart-fill heart_icon"></i> TzWind
      </li>
    </ul>
    <ul>
      <li>
        
        
        
        由 <a href="https://hexo.io" target="_blank">Hexo</a> 强力驱动
        <span class="division">|</span>
        主题 - <a href="https://github.com/Shen-Yu/hexo-theme-ayer" target="_blank">Ayer</a>
        
      </li>
    </ul>
    <ul>
      <li>
        
        
        <span>
  <span><i class="ri-user-3-fill"></i>访问人数:<span id="busuanzi_value_site_uv"></span></s>
  <span class="division">|</span>
  <span><i class="ri-eye-fill"></i>浏览次数:<span id="busuanzi_value_page_pv"></span></span>
</span>
        
      </li>
    </ul>
    <ul>
      
    </ul>
    <ul>
      
    </ul>
    <ul>
      <li>
        <!-- cnzz统计 -->
        
        <script type="text/javascript" src='https://s9.cnzz.com/z_stat.php?id=1278069914&amp;web_id=1278069914'></script>
        
      </li>
    </ul>
  </div>
</footer>
      <div class="float_btns">
        <div class="totop" id="totop">
  <i class="ri-arrow-up-line"></i>
</div>

<div class="todark" id="todark">
  <i class="ri-moon-line"></i>
</div>

      </div>
    </main>
    <aside class="sidebar on">
      <button class="navbar-toggle"></button>
<nav class="navbar">
  
  <div class="logo">
    <a href="/"><img src="/images/ayer-side.svg" alt="Hexo"></a>
  </div>
  
  <ul class="nav nav-main">
    
    <li class="nav-item">
      <a class="nav-item-link" href="/">主页</a>
    </li>
    
    <li class="nav-item">
      <a class="nav-item-link" href="/archives">归档</a>
    </li>
    
    <li class="nav-item">
      <a class="nav-item-link" href="/categories">分类</a>
    </li>
    
    <li class="nav-item">
      <a class="nav-item-link" href="/tags">标签</a>
    </li>
    
    <li class="nav-item">
      <a class="nav-item-link" target="_blank" rel="noopener" href="http://www.baidu.com">百度</a>
    </li>
    
    <li class="nav-item">
      <a class="nav-item-link" href="/friends">友链</a>
    </li>
    
    <li class="nav-item">
      <a class="nav-item-link" href="/2019/about">关于我</a>
    </li>
    
  </ul>
</nav>
<nav class="navbar navbar-bottom">
  <ul class="nav">
    <li class="nav-item">
      
      <a class="nav-item-link nav-item-search"  title="搜索">
        <i class="ri-search-line"></i>
      </a>
      
      
      <a class="nav-item-link" target="_blank" href="/atom.xml" title="RSS Feed">
        <i class="ri-rss-line"></i>
      </a>
      
    </li>
  </ul>
</nav>
<div class="search-form-wrap">
  <div class="local-search local-search-plugin">
  <input type="search" id="local-search-input" class="local-search-input" placeholder="Search...">
  <div id="local-search-result" class="local-search-result"></div>
</div>
</div>
    </aside>
    <script>
      if (window.matchMedia("(max-width: 768px)").matches) {
        document.querySelector('.content').classList.remove('on');
        document.querySelector('.sidebar').classList.remove('on');
      }
    </script>
    <div id="mask"></div>

<!-- #reward -->
<div id="reward">
  <span class="close"><i class="ri-close-line"></i></span>
  <p class="reward-p"><i class="ri-cup-line"></i>请我喝杯咖啡吧~</p>
  <div class="reward-box">
    
    
  </div>
</div>
    
<script src="/js/jquery-2.0.3.min.js"></script>


<script src="/js/lazyload.min.js"></script>

<!-- Tocbot -->


<script src="/js/tocbot.min.js"></script>

<script>
  tocbot.init({
    tocSelector: '.tocbot',
    contentSelector: '.article-entry',
    headingSelector: 'h1, h2, h3, h4, h5, h6',
    hasInnerContainers: true,
    scrollSmooth: true,
    scrollContainer: 'main',
    positionFixedSelector: '.tocbot',
    positionFixedClass: 'is-position-fixed',
    fixedSidebarOffset: 'auto'
  });
</script>

<script src="https://cdn.jsdelivr.net/npm/jquery-modal@0.9.2/jquery.modal.min.js"></script>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/jquery-modal@0.9.2/jquery.modal.min.css">
<script src="https://cdn.jsdelivr.net/npm/justifiedGallery@3.7.0/dist/js/jquery.justifiedGallery.min.js"></script>

<script src="/dist/main.js"></script>

<!-- ImageViewer -->

<!-- Root element of PhotoSwipe. Must have class pswp. -->
<div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">

    <!-- Background of PhotoSwipe. 
         It's a separate element as animating opacity is faster than rgba(). -->
    <div class="pswp__bg"></div>

    <!-- Slides wrapper with overflow:hidden. -->
    <div class="pswp__scroll-wrap">

        <!-- Container that holds slides. 
            PhotoSwipe keeps only 3 of them in the DOM to save memory.
            Don't modify these 3 pswp__item elements, data is added later on. -->
        <div class="pswp__container">
            <div class="pswp__item"></div>
            <div class="pswp__item"></div>
            <div class="pswp__item"></div>
        </div>

        <!-- Default (PhotoSwipeUI_Default) interface on top of sliding area. Can be changed. -->
        <div class="pswp__ui pswp__ui--hidden">

            <div class="pswp__top-bar">

                <!--  Controls are self-explanatory. Order can be changed. -->

                <div class="pswp__counter"></div>

                <button class="pswp__button pswp__button--close" title="Close (Esc)"></button>

                <button class="pswp__button pswp__button--share" style="display:none" title="Share"></button>

                <button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>

                <button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>

                <!-- Preloader demo http://codepen.io/dimsemenov/pen/yyBWoR -->
                <!-- element will get class pswp__preloader--active when preloader is running -->
                <div class="pswp__preloader">
                    <div class="pswp__preloader__icn">
                        <div class="pswp__preloader__cut">
                            <div class="pswp__preloader__donut"></div>
                        </div>
                    </div>
                </div>
            </div>

            <div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
                <div class="pswp__share-tooltip"></div>
            </div>

            <button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)">
            </button>

            <button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)">
            </button>

            <div class="pswp__caption">
                <div class="pswp__caption__center"></div>
            </div>

        </div>

    </div>

</div>

<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/photoswipe@4.1.3/dist/photoswipe.min.css">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/photoswipe@4.1.3/dist/default-skin/default-skin.min.css">
<script src="https://cdn.jsdelivr.net/npm/photoswipe@4.1.3/dist/photoswipe.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/photoswipe@4.1.3/dist/photoswipe-ui-default.min.js"></script>

<script>
    function viewer_init() {
        let pswpElement = document.querySelectorAll('.pswp')[0];
        let $imgArr = document.querySelectorAll(('.article-entry img:not(.reward-img)'))

        $imgArr.forEach(($em, i) => {
            $em.onclick = () => {
                // slider展开状态
                // todo: 这样不好，后面改成状态
                if (document.querySelector('.left-col.show')) return
                let items = []
                $imgArr.forEach(($em2, i2) => {
                    let img = $em2.getAttribute('data-idx', i2)
                    let src = $em2.getAttribute('data-target') || $em2.getAttribute('src')
                    let title = $em2.getAttribute('alt')
                    // 获得原图尺寸
                    const image = new Image()
                    image.src = src
                    items.push({
                        src: src,
                        w: image.width || $em2.width,
                        h: image.height || $em2.height,
                        title: title
                    })
                })
                var gallery = new PhotoSwipe(pswpElement, PhotoSwipeUI_Default, items, {
                    index: parseInt(i)
                });
                gallery.init()
            }
        })
    }
    viewer_init()
</script>

<!-- MathJax -->

<!-- Katex -->

<!-- busuanzi  -->


<script src="/js/busuanzi-2.3.pure.min.js"></script>


<!-- ClickLove -->

<!-- ClickBoom1 -->

<!-- ClickBoom2 -->

<!-- CodeCopy -->


<link rel="stylesheet" href="/css/clipboard.css">

<script src="https://cdn.jsdelivr.net/npm/clipboard@2/dist/clipboard.min.js"></script>
<script>
  function wait(callback, seconds) {
    var timelag = null;
    timelag = window.setTimeout(callback, seconds);
  }
  !function (e, t, a) {
    var initCopyCode = function(){
      var copyHtml = '';
      copyHtml += '<button class="btn-copy" data-clipboard-snippet="">';
      copyHtml += '<i class="ri-file-copy-2-line"></i><span>COPY</span>';
      copyHtml += '</button>';
      $(".highlight .code pre").before(copyHtml);
      $(".article pre code").before(copyHtml);
      var clipboard = new ClipboardJS('.btn-copy', {
        target: function(trigger) {
          return trigger.nextElementSibling;
        }
      });
      clipboard.on('success', function(e) {
        let $btn = $(e.trigger);
        $btn.addClass('copied');
        let $icon = $($btn.find('i'));
        $icon.removeClass('ri-file-copy-2-line');
        $icon.addClass('ri-checkbox-circle-line');
        let $span = $($btn.find('span'));
        $span[0].innerText = 'COPIED';
        
        wait(function () { // 等待两秒钟后恢复
          $icon.removeClass('ri-checkbox-circle-line');
          $icon.addClass('ri-file-copy-2-line');
          $span[0].innerText = 'COPY';
        }, 2000);
      });
      clipboard.on('error', function(e) {
        e.clearSelection();
        let $btn = $(e.trigger);
        $btn.addClass('copy-failed');
        let $icon = $($btn.find('i'));
        $icon.removeClass('ri-file-copy-2-line');
        $icon.addClass('ri-time-line');
        let $span = $($btn.find('span'));
        $span[0].innerText = 'COPY FAILED';
        
        wait(function () { // 等待两秒钟后恢复
          $icon.removeClass('ri-time-line');
          $icon.addClass('ri-file-copy-2-line');
          $span[0].innerText = 'COPY';
        }, 2000);
      });
    }
    initCopyCode();
  }(window, document);
</script>


<!-- CanvasBackground -->


    
  </div>
</body>

</html>